# -*- coding: utf-8 -*-

import re
import sys
import scrapy
from scrapy.linkextractors import LinkExtractor
from items import BookItem
from bs4 import BeautifulSoup

reload(sys)
sys.setdefaultencoding('utf8')


class BooksSpider(scrapy.Spider):
    name = 'books'
    allowed_domains = ['books.toscrape.com']
    start_urls = ['http://books.toscrape.com/']

    def parse(self, response):
        le = LinkExtractor(restrict_css='article.product_pod h3')
        for link in le.extract_links(response):
            yield scrapy.Request(url=link.url, callback=self.parse_book)

        le = LinkExtractor(restrict_css='ul.pager li.next')
        links = le.extract_links(response)
        if links:
            next_url = links[0].url
            yield scrapy.Request(next_url, callback=self.parse)

        pass

    def parse_book(self, response):
        book = BookItem()
        bs_obj = BeautifulSoup(response.text, "html5lib")
        book['name'] = bs_obj.find("h1").get_text()
        book['price'] = bs_obj.find("p", attrs={"class":"price_color"}).get_text()
        review_rating_tag = bs_obj.find("p", attrs={"class":re.compile("star-rating")})
        if review_rating_tag:
            review_rating = review_rating_tag.attrs['class'][1]
        else:
            review_rating = ""
        book['review_rating'] = review_rating
        book['upc'] = bs_obj.find("th", text={"UPC"}).find_next_sibling("td").get_text()
        stock_tag = bs_obj.find("th", text={"Availability"}).find_next_sibling("td")
        if stock_tag:
            stock = re.findall("\d+", stock_tag.get_text())[0]
        else:
            stock = ""
        book['stock'] = stock
        book['review_num'] = bs_obj.find("th", text={"Number of reviews"}).find_next_sibling("td").get_text()

        yield book
